{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import nltk, re, pprint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 从网络和硬盘访问文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "\n",
    "url = \"http://www.gutenberg.org/files/2554/2554-0.txt\"\n",
    "raw = urlopen(url).read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(bytes, str)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(raw), type(str(raw))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1358498"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = str(raw)\n",
    "len(raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"b'\\\\xef\\\\xbb\\\\xbfThe Project Gutenberg EBook of Crime and Punishment, by Fyodo\""
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw[:75]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = nltk.word_tokenize(raw)  # 对字符串进行分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(list, 225135)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(tokens), len(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"b'\\\\xef\\\\xbb\\\\xbfThe\",\n",
       " 'Project',\n",
       " 'Gutenberg',\n",
       " 'EBook',\n",
       " 'of',\n",
       " 'Crime',\n",
       " 'and',\n",
       " 'Punishment',\n",
       " ',',\n",
       " 'by']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nltk.text.Text"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = nltk.Text(tokens)\n",
    "type(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['man',\n",
       " 'had',\n",
       " 'a\\\\r\\\\nsick',\n",
       " ',',\n",
       " 'frightened',\n",
       " 'feeling',\n",
       " ',',\n",
       " 'which',\n",
       " 'made',\n",
       " 'him',\n",
       " 'scowl',\n",
       " 'and',\n",
       " 'feel',\n",
       " 'ashamed',\n",
       " '.',\n",
       " 'He',\n",
       " 'was\\\\r\\\\nhopelessly',\n",
       " 'in',\n",
       " 'debt',\n",
       " 'to',\n",
       " 'his',\n",
       " 'landlady',\n",
       " ',',\n",
       " 'and',\n",
       " 'was',\n",
       " 'afraid',\n",
       " 'of',\n",
       " 'meeting',\n",
       " 'her.\\\\r\\\\n\\\\r\\\\nThis',\n",
       " 'was',\n",
       " 'not',\n",
       " 'because',\n",
       " 'he',\n",
       " 'was',\n",
       " 'cowardly',\n",
       " 'and',\n",
       " 'abject',\n",
       " ',',\n",
       " 'quite',\n",
       " 'the']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text[1020: 1060]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Katerina Ivanovna; Pyotr Petrovitch; Avdotya Romanovna; Pulcheria\n",
      "Alexandrovna; Rodion Romanovitch; Marfa Petrovna; \\xe2\\x80\\x9d cried;\n",
      "Sofya Semyonovna; \\xe2\\x80\\x9d said; old woman; \\xe2\\x80\\x9d\n",
      "Raskolnikov; Porfiry Petrovitch; Project Gutenberg-tm;\n",
      "don\\xe2\\x80\\x99t know; great deal; Amalia Ivanovna; young man; Hay\n",
      "Market; police station; Nikodim Fomitch\n"
     ]
    }
   ],
   "source": [
    "text.collocations()  # 检测高频双连词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5866"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw.find(\"PART I\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1338204"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw.rfind(\"the subject of a new story\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = raw[5866: 1338204]\n",
    "raw.find(\"PART I\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = raw[5866: 1338204]\n",
    "raw.find(\"the subject of a new story\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "============================================================"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'<?xml version=\"1.0\" encoding=\"utf-8\"?>\\r\\n\\r\\n<!DOCTYPE html\\r\\n  '"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = \"http://www.gutenberg.org/files/2554/2554-h/2554-h.htm\"\n",
    "html = urlopen(url).read()\n",
    "html[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Crime',\n",
       " 'and',\n",
       " 'Punishment',\n",
       " ',',\n",
       " 'by',\n",
       " 'Fyodor',\n",
       " 'Dostoevsky',\n",
       " 'The',\n",
       " 'Project',\n",
       " 'Gutenberg',\n",
       " 'EBook',\n",
       " 'of',\n",
       " 'Crime',\n",
       " 'and',\n",
       " 'Punishment',\n",
       " ',',\n",
       " 'by',\n",
       " 'Fyodor',\n",
       " 'Dostoevsky',\n",
       " 'This',\n",
       " 'eBook',\n",
       " 'is',\n",
       " 'for',\n",
       " 'the',\n",
       " 'use',\n",
       " 'of',\n",
       " 'anyone',\n",
       " 'anywhere',\n",
       " 'at',\n",
       " 'no',\n",
       " 'cost',\n",
       " 'and',\n",
       " 'with',\n",
       " 'almost',\n",
       " 'no',\n",
       " 'restrictions',\n",
       " 'whatsoever',\n",
       " '.',\n",
       " 'You',\n",
       " 'may',\n",
       " 'copy',\n",
       " 'it',\n",
       " ',',\n",
       " 'give',\n",
       " 'it',\n",
       " 'away',\n",
       " 'or',\n",
       " 're-use',\n",
       " 'it',\n",
       " 'under',\n",
       " 'the',\n",
       " 'terms',\n",
       " 'of',\n",
       " 'the',\n",
       " 'Project',\n",
       " 'Gutenberg',\n",
       " 'License',\n",
       " 'included',\n",
       " 'with',\n",
       " 'this',\n",
       " 'eBook',\n",
       " 'or',\n",
       " 'online',\n",
       " 'at',\n",
       " 'www.gutenberg.org',\n",
       " 'Title',\n",
       " ':',\n",
       " 'Crime',\n",
       " 'and',\n",
       " 'Punishment',\n",
       " 'Author',\n",
       " ':',\n",
       " 'Fyodor',\n",
       " 'Dostoevsky',\n",
       " 'Release',\n",
       " 'Date',\n",
       " ':',\n",
       " 'March',\n",
       " '28',\n",
       " ',',\n",
       " '2006',\n",
       " '[',\n",
       " 'EBook',\n",
       " '#',\n",
       " '2554',\n",
       " ']',\n",
       " 'Last',\n",
       " 'Updated',\n",
       " ':',\n",
       " 'October',\n",
       " '27',\n",
       " ',',\n",
       " '2016',\n",
       " 'Language',\n",
       " ':',\n",
       " 'English',\n",
       " 'Character',\n",
       " 'set',\n",
       " 'encoding',\n",
       " ':',\n",
       " 'UTF-8',\n",
       " '*',\n",
       " '*',\n",
       " '*',\n",
       " 'START',\n",
       " 'OF',\n",
       " 'THIS',\n",
       " 'PROJECT',\n",
       " 'GUTENBERG',\n",
       " 'EBOOK',\n",
       " 'CRIME',\n",
       " 'AND',\n",
       " 'PUNISHMENT',\n",
       " '*',\n",
       " '*',\n",
       " '*',\n",
       " 'Produced',\n",
       " 'by',\n",
       " 'John',\n",
       " 'Bickers',\n",
       " ';',\n",
       " 'and',\n",
       " 'Dagny',\n",
       " 'and',\n",
       " 'David',\n",
       " 'Widger',\n",
       " 'CRIME',\n",
       " 'AND',\n",
       " 'PUNISHMENT',\n",
       " 'By',\n",
       " 'Fyodor',\n",
       " 'Dostoevsky',\n",
       " 'Translated',\n",
       " 'By',\n",
       " 'Constance',\n",
       " 'Garnett',\n",
       " 'CONTENTS',\n",
       " 'TRANSLATOR',\n",
       " '’',\n",
       " 'S',\n",
       " 'PREFACE',\n",
       " 'CRIME',\n",
       " 'AND',\n",
       " 'PUNISHMENT',\n",
       " 'PART',\n",
       " 'I',\n",
       " 'CHAPTER',\n",
       " 'I',\n",
       " 'CHAPTER',\n",
       " 'II',\n",
       " 'CHAPTER',\n",
       " 'III',\n",
       " 'CHAPTER',\n",
       " 'IV',\n",
       " 'CHAPTER',\n",
       " 'V',\n",
       " 'CHAPTER',\n",
       " 'VI',\n",
       " 'CHAPTER',\n",
       " 'VII',\n",
       " 'PART',\n",
       " 'II',\n",
       " 'CHAPTER',\n",
       " 'I',\n",
       " 'CHAPTER',\n",
       " 'II',\n",
       " 'CHAPTER',\n",
       " 'III',\n",
       " 'CHAPTER',\n",
       " 'IV',\n",
       " 'CHAPTER',\n",
       " 'V',\n",
       " 'CHAPTER',\n",
       " 'VI',\n",
       " 'CHAPTER',\n",
       " 'VII',\n",
       " 'PART',\n",
       " 'III',\n",
       " 'CHAPTER',\n",
       " 'I',\n",
       " 'CHAPTER',\n",
       " 'II',\n",
       " 'CHAPTER',\n",
       " 'III',\n",
       " 'CHAPTER',\n",
       " 'IV',\n",
       " 'CHAPTER',\n",
       " 'V',\n",
       " 'CHAPTER',\n",
       " 'VI',\n",
       " 'PART',\n",
       " 'IV',\n",
       " 'CHAPTER',\n",
       " 'I',\n",
       " 'CHAPTER',\n",
       " 'II',\n",
       " 'CHAPTER',\n",
       " 'III',\n",
       " 'CHAPTER',\n",
       " 'IV',\n",
       " 'CHAPTER',\n",
       " 'V',\n",
       " 'CHAPTER',\n",
       " 'VI',\n",
       " 'PART',\n",
       " 'V',\n",
       " 'CHAPTER',\n",
       " 'I',\n",
       " 'CHAPTER',\n",
       " 'II',\n",
       " 'CHAPTER',\n",
       " 'III',\n",
       " 'CHAPTER',\n",
       " 'IV',\n",
       " 'CHAPTER',\n",
       " 'V',\n",
       " 'PART',\n",
       " 'VI',\n",
       " 'CHAPTER',\n",
       " 'I',\n",
       " 'CHAPTER',\n",
       " 'II',\n",
       " 'CHAPTER',\n",
       " 'III',\n",
       " 'CHAPTER',\n",
       " 'IV',\n",
       " 'CHAPTER',\n",
       " 'V',\n",
       " 'CHAPTER',\n",
       " 'VI',\n",
       " 'CHAPTER',\n",
       " 'VII',\n",
       " 'CHAPTER',\n",
       " 'VIII',\n",
       " 'EPILOGUE',\n",
       " 'TRANSLATOR',\n",
       " '’',\n",
       " 'S',\n",
       " 'PREFACE',\n",
       " 'A',\n",
       " 'few',\n",
       " 'words',\n",
       " 'about',\n",
       " 'Dostoevsky',\n",
       " 'himself',\n",
       " 'may',\n",
       " 'help',\n",
       " 'the',\n",
       " 'English',\n",
       " 'reader',\n",
       " 'to',\n",
       " 'understand',\n",
       " 'his',\n",
       " 'work',\n",
       " '.',\n",
       " 'Dostoevsky',\n",
       " 'was',\n",
       " 'the',\n",
       " 'son',\n",
       " 'of',\n",
       " 'a',\n",
       " 'doctor',\n",
       " '.',\n",
       " 'His',\n",
       " 'parents',\n",
       " 'were',\n",
       " 'very',\n",
       " 'hard-working',\n",
       " 'and',\n",
       " 'deeply',\n",
       " 'religious',\n",
       " 'people',\n",
       " ',',\n",
       " 'but',\n",
       " 'so',\n",
       " 'poor',\n",
       " 'that',\n",
       " 'they',\n",
       " 'lived',\n",
       " 'with',\n",
       " 'their',\n",
       " 'five',\n",
       " 'children',\n",
       " 'in',\n",
       " 'only',\n",
       " 'two',\n",
       " 'rooms',\n",
       " '.',\n",
       " 'The',\n",
       " 'father',\n",
       " 'and',\n",
       " 'mother',\n",
       " 'spent',\n",
       " 'their',\n",
       " 'evenings',\n",
       " 'in',\n",
       " 'reading',\n",
       " 'aloud',\n",
       " 'to',\n",
       " 'their',\n",
       " 'children',\n",
       " ',',\n",
       " 'generally',\n",
       " 'from',\n",
       " 'books',\n",
       " 'of',\n",
       " 'a',\n",
       " 'serious',\n",
       " 'character',\n",
       " '.',\n",
       " 'Though',\n",
       " 'always',\n",
       " 'sickly',\n",
       " 'and',\n",
       " 'delicate',\n",
       " 'Dostoevsky',\n",
       " 'came',\n",
       " 'out',\n",
       " 'third',\n",
       " 'in',\n",
       " 'the',\n",
       " 'final',\n",
       " 'examination',\n",
       " 'of',\n",
       " 'the',\n",
       " 'Petersburg',\n",
       " 'school',\n",
       " 'of',\n",
       " 'Engineering',\n",
       " '.',\n",
       " 'There',\n",
       " 'he',\n",
       " 'had',\n",
       " 'already',\n",
       " 'begun',\n",
       " 'his',\n",
       " 'first',\n",
       " 'work',\n",
       " ',',\n",
       " '“',\n",
       " 'Poor',\n",
       " 'Folk.',\n",
       " '”',\n",
       " 'This',\n",
       " 'story',\n",
       " 'was',\n",
       " 'published',\n",
       " 'by',\n",
       " 'the',\n",
       " 'poet',\n",
       " 'Nekrassov',\n",
       " 'in',\n",
       " 'his',\n",
       " 'review',\n",
       " 'and',\n",
       " 'was',\n",
       " 'received',\n",
       " 'with',\n",
       " 'acclamations',\n",
       " '.',\n",
       " 'The',\n",
       " 'shy',\n",
       " ',',\n",
       " 'unknown',\n",
       " 'youth',\n",
       " 'found',\n",
       " 'himself',\n",
       " 'instantly',\n",
       " 'something',\n",
       " 'of',\n",
       " 'a',\n",
       " 'celebrity',\n",
       " '.',\n",
       " 'A',\n",
       " 'brilliant',\n",
       " 'and',\n",
       " 'successful',\n",
       " 'career',\n",
       " 'seemed',\n",
       " 'to',\n",
       " 'open',\n",
       " 'before',\n",
       " 'him',\n",
       " ',',\n",
       " 'but',\n",
       " 'those',\n",
       " 'hopes',\n",
       " 'were',\n",
       " 'soon',\n",
       " 'dashed',\n",
       " '.',\n",
       " 'In',\n",
       " '1849',\n",
       " 'he',\n",
       " 'was',\n",
       " 'arrested',\n",
       " '.',\n",
       " 'Though',\n",
       " 'neither',\n",
       " 'by',\n",
       " 'temperament',\n",
       " 'nor',\n",
       " 'conviction',\n",
       " 'a',\n",
       " 'revolutionist',\n",
       " ',',\n",
       " 'Dostoevsky',\n",
       " 'was',\n",
       " 'one',\n",
       " 'of',\n",
       " 'a',\n",
       " 'little',\n",
       " 'group',\n",
       " 'of',\n",
       " 'young',\n",
       " 'men',\n",
       " 'who',\n",
       " 'met',\n",
       " 'together',\n",
       " 'to',\n",
       " 'read',\n",
       " 'Fourier',\n",
       " 'and',\n",
       " 'Proudhon',\n",
       " '.',\n",
       " 'He',\n",
       " 'was',\n",
       " 'accused',\n",
       " 'of',\n",
       " '“',\n",
       " 'taking',\n",
       " 'part',\n",
       " 'in',\n",
       " 'conversations',\n",
       " 'against',\n",
       " 'the',\n",
       " 'censorship',\n",
       " ',',\n",
       " 'of',\n",
       " 'reading',\n",
       " 'a',\n",
       " 'letter',\n",
       " 'from',\n",
       " 'Byelinsky',\n",
       " 'to',\n",
       " 'Gogol',\n",
       " ',',\n",
       " 'and',\n",
       " 'of',\n",
       " 'knowing',\n",
       " 'of',\n",
       " 'the',\n",
       " 'intention',\n",
       " 'to',\n",
       " 'set',\n",
       " 'up',\n",
       " 'a',\n",
       " 'printing',\n",
       " 'press.',\n",
       " '”',\n",
       " 'Under',\n",
       " 'Nicholas',\n",
       " 'I',\n",
       " '.',\n",
       " '(',\n",
       " 'that',\n",
       " '“',\n",
       " 'stern',\n",
       " 'and',\n",
       " 'just',\n",
       " 'man',\n",
       " ',',\n",
       " '”',\n",
       " 'as',\n",
       " 'Maurice',\n",
       " 'Baring',\n",
       " 'calls',\n",
       " 'him',\n",
       " ')',\n",
       " 'this',\n",
       " 'was',\n",
       " 'enough',\n",
       " ',',\n",
       " 'and',\n",
       " 'he',\n",
       " 'was',\n",
       " 'condemned',\n",
       " 'to',\n",
       " 'death',\n",
       " '.',\n",
       " 'After',\n",
       " 'eight',\n",
       " 'months',\n",
       " '’',\n",
       " 'imprisonment',\n",
       " 'he',\n",
       " 'was',\n",
       " 'with',\n",
       " 'twenty-one',\n",
       " 'others',\n",
       " 'taken',\n",
       " 'out',\n",
       " 'to',\n",
       " 'the',\n",
       " 'Semyonovsky',\n",
       " 'Square',\n",
       " 'to',\n",
       " 'be',\n",
       " 'shot',\n",
       " '.',\n",
       " 'Writing',\n",
       " 'to',\n",
       " 'his',\n",
       " 'brother',\n",
       " 'Mihail',\n",
       " ',',\n",
       " 'Dostoevsky',\n",
       " 'says',\n",
       " ':',\n",
       " '“',\n",
       " 'They',\n",
       " 'snapped',\n",
       " 'words',\n",
       " 'over',\n",
       " 'our',\n",
       " 'heads',\n",
       " ',',\n",
       " 'and',\n",
       " 'they',\n",
       " 'made',\n",
       " 'us',\n",
       " 'put',\n",
       " 'on',\n",
       " 'the',\n",
       " 'white',\n",
       " 'shirts',\n",
       " 'worn',\n",
       " 'by',\n",
       " 'persons',\n",
       " 'condemned',\n",
       " 'to',\n",
       " 'death',\n",
       " '.',\n",
       " 'Thereupon',\n",
       " 'we',\n",
       " 'were',\n",
       " 'bound',\n",
       " 'in',\n",
       " 'threes',\n",
       " 'to',\n",
       " 'stakes',\n",
       " ',',\n",
       " 'to',\n",
       " 'suffer',\n",
       " 'execution',\n",
       " '.',\n",
       " 'Being',\n",
       " 'the',\n",
       " 'third',\n",
       " 'in',\n",
       " 'the',\n",
       " 'row',\n",
       " ',',\n",
       " 'I',\n",
       " 'concluded',\n",
       " 'I',\n",
       " 'had',\n",
       " 'only',\n",
       " 'a',\n",
       " 'few',\n",
       " 'minutes',\n",
       " 'of',\n",
       " 'life',\n",
       " 'before',\n",
       " 'me',\n",
       " '.',\n",
       " 'I',\n",
       " 'thought',\n",
       " 'of',\n",
       " 'you',\n",
       " 'and',\n",
       " 'your',\n",
       " 'dear',\n",
       " 'ones',\n",
       " 'and',\n",
       " 'I',\n",
       " 'contrived',\n",
       " 'to',\n",
       " 'kiss',\n",
       " 'Plestcheiev',\n",
       " 'and',\n",
       " 'Dourov',\n",
       " ',',\n",
       " 'who',\n",
       " 'were',\n",
       " 'next',\n",
       " 'to',\n",
       " 'me',\n",
       " ',',\n",
       " 'and',\n",
       " 'to',\n",
       " 'bid',\n",
       " 'them',\n",
       " 'farewell',\n",
       " '.',\n",
       " 'Suddenly',\n",
       " 'the',\n",
       " 'troops',\n",
       " 'beat',\n",
       " 'a',\n",
       " 'tattoo',\n",
       " ',',\n",
       " 'we',\n",
       " 'were',\n",
       " 'unbound',\n",
       " ',',\n",
       " 'brought',\n",
       " 'back',\n",
       " 'upon',\n",
       " 'the',\n",
       " 'scaffold',\n",
       " ',',\n",
       " 'and',\n",
       " 'informed',\n",
       " 'that',\n",
       " 'his',\n",
       " 'Majesty',\n",
       " 'had',\n",
       " 'spared',\n",
       " 'us',\n",
       " 'our',\n",
       " 'lives.',\n",
       " '”',\n",
       " 'The',\n",
       " 'sentence',\n",
       " 'was',\n",
       " 'commuted',\n",
       " 'to',\n",
       " 'hard',\n",
       " 'labour',\n",
       " '.',\n",
       " 'One',\n",
       " 'of',\n",
       " 'the',\n",
       " 'prisoners',\n",
       " ',',\n",
       " 'Grigoryev',\n",
       " ',',\n",
       " 'went',\n",
       " 'mad',\n",
       " 'as',\n",
       " 'soon',\n",
       " 'as',\n",
       " 'he',\n",
       " 'was',\n",
       " 'untied',\n",
       " ',',\n",
       " 'and',\n",
       " 'never',\n",
       " 'regained',\n",
       " 'his',\n",
       " 'sanity',\n",
       " '.',\n",
       " 'The',\n",
       " 'intense',\n",
       " 'suffering',\n",
       " 'of',\n",
       " 'this',\n",
       " 'experience',\n",
       " 'left',\n",
       " 'a',\n",
       " 'lasting',\n",
       " 'stamp',\n",
       " 'on',\n",
       " 'Dostoevsky',\n",
       " '’',\n",
       " 's',\n",
       " 'mind',\n",
       " '.',\n",
       " 'Though',\n",
       " 'his',\n",
       " 'religious',\n",
       " 'temper',\n",
       " 'led',\n",
       " 'him',\n",
       " 'in',\n",
       " 'the',\n",
       " 'end',\n",
       " 'to',\n",
       " 'accept',\n",
       " 'every',\n",
       " 'suffering',\n",
       " 'with',\n",
       " 'resignation',\n",
       " 'and',\n",
       " 'to',\n",
       " 'regard',\n",
       " 'it',\n",
       " 'as',\n",
       " 'a',\n",
       " 'blessing',\n",
       " 'in',\n",
       " 'his',\n",
       " 'own',\n",
       " 'case',\n",
       " ',',\n",
       " 'he',\n",
       " 'constantly',\n",
       " 'recurs',\n",
       " 'to',\n",
       " 'the',\n",
       " 'subject',\n",
       " 'in',\n",
       " 'his',\n",
       " 'writings',\n",
       " '.',\n",
       " 'He',\n",
       " 'describes',\n",
       " 'the',\n",
       " 'awful',\n",
       " 'agony',\n",
       " 'of',\n",
       " 'the',\n",
       " 'condemned',\n",
       " 'man',\n",
       " 'and',\n",
       " 'insists',\n",
       " 'on',\n",
       " 'the',\n",
       " 'cruelty',\n",
       " 'of',\n",
       " 'inflicting',\n",
       " 'such',\n",
       " 'torture',\n",
       " '.',\n",
       " 'Then',\n",
       " 'followed',\n",
       " 'four',\n",
       " 'years',\n",
       " 'of',\n",
       " 'penal',\n",
       " 'servitude',\n",
       " ',',\n",
       " 'spent',\n",
       " 'in',\n",
       " 'the',\n",
       " 'company',\n",
       " 'of',\n",
       " 'common',\n",
       " 'criminals',\n",
       " 'in',\n",
       " 'Siberia',\n",
       " ',',\n",
       " 'where',\n",
       " 'he',\n",
       " 'began',\n",
       " 'the',\n",
       " '“',\n",
       " 'Dead',\n",
       " 'House',\n",
       " ',',\n",
       " '”',\n",
       " 'and',\n",
       " 'some',\n",
       " 'years',\n",
       " 'of',\n",
       " 'service',\n",
       " 'in',\n",
       " 'a',\n",
       " 'disciplinary',\n",
       " 'battalion',\n",
       " '.',\n",
       " 'He',\n",
       " 'had',\n",
       " 'shown',\n",
       " 'signs',\n",
       " 'of',\n",
       " 'some',\n",
       " 'obscure',\n",
       " 'nervous',\n",
       " 'disease',\n",
       " 'before',\n",
       " 'his',\n",
       " 'arrest',\n",
       " 'and',\n",
       " 'this',\n",
       " 'now',\n",
       " 'developed',\n",
       " 'into',\n",
       " 'violent',\n",
       " 'attacks',\n",
       " 'of',\n",
       " 'epilepsy',\n",
       " ',',\n",
       " 'from',\n",
       " 'which',\n",
       " 'he',\n",
       " 'suffered',\n",
       " 'for',\n",
       " 'the',\n",
       " 'rest',\n",
       " 'of',\n",
       " 'his',\n",
       " 'life',\n",
       " '.',\n",
       " 'The',\n",
       " 'fits',\n",
       " 'occurred',\n",
       " 'three',\n",
       " 'or',\n",
       " 'four',\n",
       " 'times',\n",
       " 'a',\n",
       " 'year',\n",
       " 'and',\n",
       " 'were',\n",
       " 'more',\n",
       " 'frequent',\n",
       " 'in',\n",
       " 'periods',\n",
       " 'of',\n",
       " 'great',\n",
       " 'strain',\n",
       " '.',\n",
       " 'In',\n",
       " '1859',\n",
       " 'he',\n",
       " 'was',\n",
       " 'allowed',\n",
       " 'to',\n",
       " 'return',\n",
       " 'to',\n",
       " 'Russia',\n",
       " '.',\n",
       " 'He',\n",
       " 'started',\n",
       " 'a',\n",
       " 'journal—',\n",
       " '“',\n",
       " 'Vremya',\n",
       " ',',\n",
       " '”',\n",
       " 'which',\n",
       " 'was',\n",
       " 'forbidden',\n",
       " 'by',\n",
       " 'the',\n",
       " 'Censorship',\n",
       " 'through',\n",
       " 'a',\n",
       " 'misunderstanding',\n",
       " '.',\n",
       " 'In',\n",
       " '1864',\n",
       " 'he',\n",
       " 'lost',\n",
       " 'his',\n",
       " 'first',\n",
       " 'wife',\n",
       " 'and',\n",
       " 'his',\n",
       " 'brother',\n",
       " 'Mihail',\n",
       " '.',\n",
       " 'He',\n",
       " 'was',\n",
       " 'in',\n",
       " 'terrible',\n",
       " 'poverty',\n",
       " ',',\n",
       " 'yet',\n",
       " 'he',\n",
       " 'took',\n",
       " 'upon',\n",
       " 'himself',\n",
       " 'the',\n",
       " 'payment',\n",
       " 'of',\n",
       " 'his',\n",
       " 'brother',\n",
       " '’',\n",
       " 's',\n",
       " 'debts',\n",
       " '.',\n",
       " 'He',\n",
       " 'started',\n",
       " 'another',\n",
       " 'journal—',\n",
       " '“',\n",
       " 'The',\n",
       " 'Epoch',\n",
       " ',',\n",
       " '”',\n",
       " 'which',\n",
       " 'within',\n",
       " 'a',\n",
       " 'few',\n",
       " 'months',\n",
       " 'was',\n",
       " 'also',\n",
       " 'prohibited',\n",
       " '.',\n",
       " 'He',\n",
       " 'was',\n",
       " 'weighed',\n",
       " 'down',\n",
       " 'by',\n",
       " 'debt',\n",
       " ',',\n",
       " 'his',\n",
       " 'brother',\n",
       " '’',\n",
       " 's',\n",
       " 'family',\n",
       " 'was',\n",
       " 'dependent',\n",
       " 'on',\n",
       " 'him',\n",
       " ',',\n",
       " 'he',\n",
       " 'was',\n",
       " 'forced',\n",
       " 'to',\n",
       " 'write',\n",
       " 'at',\n",
       " 'heart-breaking',\n",
       " 'speed',\n",
       " ',',\n",
       " 'and',\n",
       " 'is',\n",
       " 'said',\n",
       " 'never',\n",
       " 'to',\n",
       " 'have',\n",
       " 'corrected',\n",
       " 'his',\n",
       " 'work',\n",
       " '.',\n",
       " 'The',\n",
       " 'later',\n",
       " 'years',\n",
       " 'of',\n",
       " 'his',\n",
       " 'life',\n",
       " 'were',\n",
       " 'much',\n",
       " 'softened',\n",
       " 'by',\n",
       " 'the',\n",
       " 'tenderness',\n",
       " 'and',\n",
       " 'devotion',\n",
       " 'of',\n",
       " 'his',\n",
       " 'second',\n",
       " 'wife',\n",
       " '.',\n",
       " 'In',\n",
       " 'June',\n",
       " '1880',\n",
       " 'he',\n",
       " 'made',\n",
       " 'his',\n",
       " 'famous',\n",
       " 'speech',\n",
       " 'at',\n",
       " 'the',\n",
       " 'unveiling',\n",
       " 'of',\n",
       " 'the',\n",
       " 'monument',\n",
       " 'to',\n",
       " 'Pushkin',\n",
       " 'in',\n",
       " 'Moscow',\n",
       " 'and',\n",
       " 'he',\n",
       " 'was',\n",
       " 'received',\n",
       " 'with',\n",
       " 'extraordinary',\n",
       " 'demonstrations',\n",
       " 'of',\n",
       " 'love',\n",
       " 'and',\n",
       " 'honour',\n",
       " '.',\n",
       " 'A',\n",
       " 'few',\n",
       " 'months',\n",
       " 'later',\n",
       " 'Dostoevsky',\n",
       " 'died',\n",
       " '.',\n",
       " 'He',\n",
       " 'was',\n",
       " 'followed',\n",
       " 'to',\n",
       " 'the',\n",
       " 'grave',\n",
       " ...]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "raw = BeautifulSoup(html).get_text()\n",
    "tokens = nltk.word_tokenize(raw)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 3 of 3 matches:\n",
      " a journal— “ Vremya , ” which was forbidden by the Censorship through a misund\n",
      "d the other day that compassion is forbidden nowadays by science itself , and t\n",
      "at do you want , fool ? ” “ It ’ s forbidden in the streets . You mustn ’ t mak\n"
     ]
    }
   ],
   "source": [
    "text = nltk.Text(tokens)\n",
    "text.concordance('forbidden')  # 检索指定词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " 'my_text.txt',\n",
       " '加工原料文本.ipynb',\n",
       " '获得文本语料和词汇资料.ipynb',\n",
       " '语言处理与Python.ipynb']"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os \n",
    "os.listdir('.')  # 检索当前目录内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取NLTK语料库文件\n",
    "path = nltk.data.find('corpora/gutenberg/melville-moby_dick.txt')\n",
    "raw = open(path, 'r').read() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "==========================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Pruska Biblioteka Państwowa. Jej dawne zbiory znane pod nazwą\\n\"Berlinka\" to skarb kultury i sztuki niemieckiej. Przewiezione przez\\nNiemców pod koniec II wojny światowej na Dolny Śląsk, zostały\\nodnalezione po 1945 r. na terytorium Polski. Trafiły do Biblioteki\\nJagiellońskiej w Krakowie, obejmują ponad 500 tys. zabytkowych\\narchiwaliów, m.in. manuskrypty Goethego, Mozarta, Beethovena, Bacha.\\n'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = nltk.data.find('corpora/unicode_samples/polish-lat2.txt')\n",
    "f = open(path, encoding='latin2')\n",
    "f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ord('a')  # 查找字符整数序数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"b'\\\\xc5\\\\x84'\""
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = u'\\u0144'\n",
    "s_utf = s.encode('utf8')  # 编码\n",
    "repr(s_utf)  # 输出UTF-8转义序列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ń\n"
     ]
    }
   ],
   "source": [
    "print(s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 正则表达式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a',\n",
       " 'aa',\n",
       " 'aal',\n",
       " 'aalii',\n",
       " 'aam',\n",
       " 'aardvark',\n",
       " 'aardwolf',\n",
       " 'aba',\n",
       " 'abac',\n",
       " 'abaca',\n",
       " 'abacate',\n",
       " 'abacay',\n",
       " 'abacinate',\n",
       " 'abacination',\n",
       " 'abaciscus',\n",
       " 'abacist',\n",
       " 'aback',\n",
       " 'abactinal',\n",
       " 'abactinally',\n",
       " 'abaction']"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordlist = [w for w in nltk.corpus.words.words() if w.islower()]\n",
    "wordlist[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abaissed',\n",
       " 'abandoned',\n",
       " 'abased',\n",
       " 'abashed',\n",
       " 'abatised',\n",
       " 'abed',\n",
       " 'aborted',\n",
       " 'abridged',\n",
       " 'abscessed',\n",
       " 'absconded']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wordlist if re.search('ed$', w)][:10]  # 查找以ed结尾的词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abjectly',\n",
       " 'adjuster',\n",
       " 'dejected',\n",
       " 'dejectly',\n",
       " 'injector',\n",
       " 'majestic',\n",
       " 'objectee',\n",
       " 'objector',\n",
       " 'rejecter',\n",
       " 'rejector']"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wordlist if re.search('^..j..t..$', w)][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abkar',\n",
       " 'abkari',\n",
       " 'acker',\n",
       " 'ackey',\n",
       " 'ackman',\n",
       " 'acknow',\n",
       " 'acknowledge',\n",
       " 'acknowledgeable',\n",
       " 'acknowledged',\n",
       " 'acknowledgedly']"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in wordlist if re.search('^a(b|c)k', w)][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['u', 'i', 'i', 'i']"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word = 'kfbhfuicbhflchbirlblirb'\n",
    "re.findall(r'[aeiou]', word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2019, 12, 31]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[int(n) for n in re.findall(r'[0-9]{2,4}', '2019-12-31')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = r'^[aeiouAEIOU]+|[aeiouAEIOU]+$|[^aeiouAEIOU]'\n",
    "def compress(word):\n",
    "    pieces = re.findall(reg, word)\n",
    "    return ''.join(pieces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Universal',\n",
       " 'Declaration',\n",
       " 'of',\n",
       " 'Human',\n",
       " 'Rights',\n",
       " 'Preamble',\n",
       " 'Whereas',\n",
       " 'recognition',\n",
       " 'of',\n",
       " 'the']"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "english_udhr = nltk.corpus.udhr.words('English-Latin1')  # 世界人权语料库\n",
    "temp = [w for w in english_udhr][:10]\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['U', 'n', 'v', 'r', 's', 'l']"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(reg, 'Universal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Unvrsl',\n",
       " 'Dclrtn',\n",
       " 'of',\n",
       " 'Hmn',\n",
       " 'Rghts',\n",
       " 'Prmble',\n",
       " 'Whrs',\n",
       " 'rcgntn',\n",
       " 'of',\n",
       " 'the']"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp = [compress(w) for w in temp]\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Unvrsl Dclrtn of Hmn Rghts Prmble Whrs rcgntn of the'"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.tokenwrap(temp)  # 组成token字符串"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Universal',\n",
       " 'Declaration',\n",
       " 'of',\n",
       " 'Human',\n",
       " 'Rights',\n",
       " 'Preamble',\n",
       " 'Whereas',\n",
       " 'recognition',\n",
       " 'of',\n",
       " 'the']"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "english_udhr[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ve', 'sa', 'ra', 'ti', 're', 're', 're', 'ti']"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvs = [cv for w in english_udhr[:10]\n",
    "              for cv in re.findall(r'[ptksvr][aeiou]', w)]\n",
    "cvs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<ConditionalFreqDist with 4 conditions>"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cfd = nltk.ConditionalFreqDist(cvs)\n",
    "cfd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  a e i \n",
      "r 1 3 0 \n",
      "s 1 0 0 \n",
      "t 0 0 2 \n",
      "v 0 1 0 \n"
     ]
    }
   ],
   "source": [
    "cfd.tabulate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_word_pairs = [(cv, w) for w in nltk.corpus.toolbox.words('rotokas.dic')\n",
    "                            for cv in re.findall(r'[ptksvr][aeiou]', w)]\n",
    "cv_index = nltk.Index(cv_word_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['kaapo',\n",
       " 'kaapopato',\n",
       " 'kaipori',\n",
       " 'kaiporipie',\n",
       " 'kaiporivira',\n",
       " 'kapo',\n",
       " 'kapoa',\n",
       " 'kapokao',\n",
       " 'kapokapo',\n",
       " 'kapokapo']"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_index['po'][:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 搜索已分词文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Text: Moby Dick by Herman Melville 1851>"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import gutenberg, nps_chat\n",
    "moby = nltk.Text(gutenberg.words('melville-moby_dick.txt'))\n",
    "moby"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a man; a man; a man; a man; a man; a man; a man; a man; a man; a man;\n",
      "a man; a man; a man; a man; a man; a man; a man; a man; a man; a man;\n",
      "a man; a man; a man; a man; a man; a man; a man; a man; a man; a man;\n",
      "a man; a man; a man; a man; a man; a man; a man; a man; a man; a man;\n",
      "a man; a man; a man; a man; a man; a man; a man; a man; a man; a man;\n",
      "a man; a man; a man; a man; a man; a man; a man; a man; a man; a man;\n",
      "a man; a man\n"
     ]
    }
   ],
   "source": [
    "# 尖括号用于标记标识符的边界，\n",
    "# 尖括号之间的所有空白都被忽略（这只对nltk中的findall()方法有效）\n",
    "moby.findall(r'<a><man>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a monied man; a nervous man; a dangerous man; a white man; a white\n",
      "man; a white man; a pious man; a queer man; a good man; a mature man;\n",
      "a white man; a Cape man; a great man; a wise man; a wise man; a\n",
      "butterless man; a white man; a fiendish man; a pale man; a furious\n",
      "man; a better man; a certain man; a complete man; a dismasted man; a\n",
      "younger man; a brave man; a brave man; a brave man; a brave man\n"
     ]
    }
   ],
   "source": [
    "moby.findall(r'<a><.*><man>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "monied; nervous; dangerous; white; white; white; pious; queer; good;\n",
      "mature; white; Cape; great; wise; wise; butterless; white; fiendish;\n",
      "pale; furious; better; certain; complete; dismasted; younger; brave;\n",
      "brave; brave; brave\n"
     ]
    }
   ],
   "source": [
    "moby.findall(r'<a>(<.*>)<man>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['now', 'im', 'left', 'with', 'this', 'gay', 'name', ...]"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w = nps_chat.words()\n",
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "you rule bro; telling you bro; u twizted bro\n"
     ]
    }
   ],
   "source": [
    "# 找出以'bro'结尾的三个词组成的短语\n",
    "chat = nltk.Text(w)\n",
    "chat.findall(r'<.*><.*><bro>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lol lol lol; lmao lol lol; lol lol lol; la la la la la; la la la; la\n",
      "la la; lovely lol lol love; lol lol lol.; la la la; la la la\n"
     ]
    }
   ],
   "source": [
    "# 找出以字母'l'开始的三个或更多词组成的序列\n",
    "chat.findall(r'<l.*>{3,}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{a}aasfgsgdha{b}{b}{b}dsdg\n"
     ]
    }
   ],
   "source": [
    "# 标记字符串指定模式\n",
    "nltk.re_show(r'^a|b', 'aaasfgsgdhabbbdsdg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 提供正则匹配的图形化程序\n",
    "nltk.app.nemo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "speed and other activities; water and other liquids; tomb and other\n",
      "landmarks; Statues and other monuments; pearls and other jewels;\n",
      "charts and other items; roads and other features; figures and other\n",
      "objects; military and other areas; demands and other factors;\n",
      "abstracts and other compilations; iron and other metals\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import brown\n",
    "hobbies_learned = nltk.Text(brown.words(categories=['hobbies', 'learned']))\n",
    "hobbies_learned.findall(r'<\\w*><and><other><\\w*s>')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 词干提取器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'word', 'or', 'unit', 'of', 'text', 'to', 'be', 'carried', 'over']"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = '''\n",
    "a word or unit of text to be carried over to a new line automatically as the margin is reached, or to fit around embedded features such as pictures.\n",
    "'''\n",
    "tokens = nltk.word_tokenize(raw)\n",
    "tokens[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "porter = nltk.PorterStemmer()\n",
    "lancaster = nltk.LancasterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'word', 'or', 'unit', 'of', 'text', 'to', 'be', 'carri', 'over']"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[porter.stem(t) for t in tokens][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'word', 'or', 'unit', 'of', 'text', 'to', 'be', 'carry', 'ov']"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[lancaster.stem(t) for t in tokens][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用词干提取器索引文本\n",
    "class IndexedText(object):\n",
    "    def __init__(self, stemmer, text):\n",
    "        self._text = text\n",
    "        self._stemmer = stemmer\n",
    "        self._index = nltk.Index((self._stem(word), i) \n",
    "                                 for (i, word) in enumerate(text))\n",
    "    \n",
    "    def concordance(self, word, width=40):\n",
    "        key = self._stem(word)\n",
    "        wc = width // 4  # 前后各多少词\n",
    "        for i in self._index[key]:  # 目的是对齐输出显示\n",
    "            lcontext = ' '.join(self._text[i-wc:i])\n",
    "            rcontext = ' '.join(self._text[i:i+wc])\n",
    "            ldisplay = '%*s' % (width, lcontext[-width:])\n",
    "            rdisplay = '%-*s' % (width, rcontext[:width])\n",
    "            print(ldisplay, rdisplay)\n",
    "            \n",
    "    def _stem(self, word):\n",
    "        return self._stemmer.stem(word).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r king ! DENNIS : Listen , strange women lying in ponds distributing swords is no\n",
      " beat a very brave retreat . ROBIN : All lies ! MINSTREL : [ singing ] Bravest of\n",
      "       Nay . Nay . Come . Come . You may lie here . Oh , but you are wounded !   \n",
      "doctors immediately ! No , no , please ! Lie down . [ clap clap ] PIGLET : Well  \n",
      "ere is much danger , for beyond the cave lies the Gorge of Eternal Peril , which \n",
      "   you . Oh ... TIM : To the north there lies a cave -- the cave of Caerbannog --\n",
      "h it and lived ! Bones of full fifty men lie strewn about its lair . So , brave k\n",
      "not stop our fight ' til each one of you lies dead , and the Holy Grail returns t\n"
     ]
    }
   ],
   "source": [
    "poter = nltk.PorterStemmer()\n",
    "grail = nltk.corpus.webtext.words('grail.txt')\n",
    "text = IndexedText(porter, grail)\n",
    "text.concordance('lie')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 词形归并"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'women', 'or', 'unit', 'of', 'text', 'to', 'be', 'carried', 'lying']"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# WordNet词形归并器删除词缀产生的词都是在它的字典中的词\n",
    "raw = '''\n",
    "a women or unit of text to be carried lying to a new line automatically as the margin is reached, or to fit around embedded features such as pictures.\n",
    "'''\n",
    "tokens = nltk.word_tokenize(raw)\n",
    "tokens[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'woman', 'or', 'unit', 'of', 'text', 'to', 'be', 'carried', 'lying']"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wnl = nltk.WordNetLemmatizer()\n",
    "[wnl.lemmatize(t) for t in tokens][:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 断句"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['In the wild events which were to follow this girl had no\\n'\n",
      " 'part at all; he never saw her again until all his tale was over.',\n",
      " 'And yet, in some indescribable way, she kept recurring like a\\n'\n",
      " 'motive in music through all his mad adventures afterwards, and the\\n'\n",
      " 'glory of her strange hair ran like a red thread through those dark\\n'\n",
      " 'and ill-drawn tapestries of the night.',\n",
      " 'For what followed was so\\nimprobable, that it might well have been a dream.',\n",
      " 'When Syme went out into the starlit street, he found it for the\\n'\n",
      " 'moment empty.',\n",
      " 'Then he realised (in some odd way) that the silence\\n'\n",
      " 'was rather a living silence than a dead one.',\n",
      " 'Directly outside the\\n'\n",
      " 'door stood a street lamp, whose gleam gilded the leaves of the tree\\n'\n",
      " 'that bent out over the fence behind him.',\n",
      " 'About a foot from the\\n'\n",
      " 'lamp-post stood a figure almost as rigid and motionless as the\\n'\n",
      " 'lamp-post itself.',\n",
      " 'The tall hat and long frock coat were black; the\\n'\n",
      " 'face, in an abrupt shadow, was almost as dark.',\n",
      " 'Only a fringe of\\n'\n",
      " 'fiery hair against the light, and also something aggressive in the\\n'\n",
      " 'attitude, proclaimed that it was the poet Gregory.',\n",
      " 'He had something\\n'\n",
      " 'of the look of a masked bravo waiting sword in hand for his foe.']\n"
     ]
    }
   ],
   "source": [
    "sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')  \n",
    "text = nltk.corpus.gutenberg.raw('chesterton-thursday.txt')\n",
    "sents = sent_tokenizer.tokenize(text)\n",
    "pprint.pprint(sents[171: 181])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = 'doyouseethekittyseethedoggydoyoulikethekittylikethedoggy'\n",
    "seg1 = '000000000000000100000000001000000000000000010000000000'\n",
    "seg2 = '010010010010000100100100001010010001001000010001001000'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "def segment(text, segs):\n",
    "    words = []\n",
    "    last = 0\n",
    "    for i in range(len(segs)):\n",
    "        if segs[i] == '1':\n",
    "            words.append(text[last: i+1])\n",
    "            last = i + 1\n",
    "    words.append(text[last:])\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "segment(text, seg1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['do',\n",
       " 'you',\n",
       " 'see',\n",
       " 'the',\n",
       " 'kitty',\n",
       " 'see',\n",
       " 'the',\n",
       " 'doggy',\n",
       " 'do',\n",
       " 'you',\n",
       " 'like',\n",
       " 'the',\n",
       " 'kitty',\n",
       " 'like',\n",
       " 'the',\n",
       " 'doggy']"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "segment(text, seg2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 通过合计每个词项与推导表的字符数，作为分词质量的得分。值越小越好\n",
    "def evaluate(text, segs):\n",
    "    words = segment(text, segs)\n",
    "    text_size = len(words)\n",
    "    lexicon_size = len(' '.join(list(set(words))))  # 查找不重复的所有词\n",
    "    return text_size + lexicon_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(text, seg1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(text, seg2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用模拟退火算法的非确定形搜索\n",
    "# 一开始仅搜索短语分词：随机扰动1/0；\n",
    "# 它们与“温度”成正比，每次迭代温度都会降低，扰动边界会减少\n",
    "from random import randint\n",
    "\n",
    "def flip(segs, pos):  # 更改pos位置的数字：0到1或1到0\n",
    "    return segs[:pos] + str(1-int(segs[pos])) + segs[pos+1:]\n",
    "\n",
    "def flip_n(segs, n):  # 对字符串随机更改n位\n",
    "    for i in range(n):\n",
    "        segs = flip(segs, randint(0, len(segs) - 1))\n",
    "    return segs\n",
    "\n",
    "def anneal(text, segs, iterations, cooling_rate):\n",
    "    temperature = float(len(segs))\n",
    "    while temperature > 0.5:\n",
    "        best_segs, best = segs, evaluate(text, segs)\n",
    "        for i in range(iterations):\n",
    "            guess = flip_n(segs, int(round(temperature)))  # 产生一个随机更改n位的串\n",
    "            score = evaluate(text, guess)\n",
    "            if score < best:\n",
    "                best, best_segs = score, guess\n",
    "        score, segs = best, best_segs\n",
    "        temperature = temperature / cooling_rate  # 更改位数递减\n",
    "        print(evaluate(text, segs), segment(text, segs))\n",
    "    print()\n",
    "    return segs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "63 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "63 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "63 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "63 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "63 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "63 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "63 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "63 ['doyouseethekitty', 'seethedoggy', 'doyoulikethekitty', 'likethedoggy']\n",
      "59 ['doyouseethekittyseet', 'hedoggy', 'doyoulik', 'ethekitty', 'liket', 'hedoggy']\n",
      "59 ['doyouseethekittyseet', 'hedoggy', 'doyoulik', 'ethekitty', 'liket', 'hedoggy']\n",
      "57 ['doyouseethekittysee', 't', 'hedoggy', 'doyoulikethekittyliket', 'hedoggy']\n",
      "57 ['doyouseethekittysee', 't', 'hedoggy', 'doyoulikethekittyliket', 'hedoggy']\n",
      "57 ['doyouseethekittysee', 't', 'hedoggy', 'doyoulikethekittyliket', 'hedoggy']\n",
      "56 ['doyouseetheki', 'ttysee', 'thedoggy', 'doyoulikethekittylike', 'thedoggy']\n",
      "54 ['doyou', 'seetheki', 'ttysee', 'thedoggy', 'doyou', 'likethekittylike', 'thedoggy']\n",
      "52 ['doyou', 'seethekittysee', 'thedoggy', 'doyou', 'likethekittylike', 'thedoggy']\n",
      "52 ['doyou', 'seethekittysee', 'thedoggy', 'doyou', 'likethekittylike', 'thedoggy']\n",
      "51 ['doyou', 'seethekittysee', 'thedoggy', 'doyou', 'like', 'thekitty', 'like', 'thedoggy']\n",
      "42 ['doyou', 'see', 'thekitty', 'see', 'thedoggy', 'doyou', 'like', 'thekitty', 'like', 'thedoggy']\n",
      "42 ['doyou', 'see', 'thekitty', 'see', 'thedoggy', 'doyou', 'like', 'thekitty', 'like', 'thedoggy']\n",
      "42 ['doyou', 'see', 'thekitty', 'see', 'thedoggy', 'doyou', 'like', 'thekitty', 'like', 'thedoggy']\n",
      "42 ['doyou', 'see', 'thekitty', 'see', 'thedoggy', 'doyou', 'like', 'thekitty', 'like', 'thedoggy']\n",
      "42 ['doyou', 'see', 'thekitty', 'see', 'thedoggy', 'doyou', 'like', 'thekitty', 'like', 'thedoggy']\n",
      "42 ['doyou', 'see', 'thekitty', 'see', 'thedoggy', 'doyou', 'like', 'thekitty', 'like', 'thedoggy']\n",
      "42 ['doyou', 'see', 'thekitty', 'see', 'thedoggy', 'doyou', 'like', 'thekitty', 'like', 'thedoggy']\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'000010010000000100100000001000010001000000010001000000'"
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "anneal(text, seg1, 5000, 1.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
